﻿using System;
using System.Collections.Generic;
using System.Text;
using System.IO;

namespace ReBasic.Algorithm
{
    public class SentenceExtractor
    {
        public event MainFrame.TripleHandler ProcessEvent;
        public event MainFrame.InfoHandler StateEvent;

        string oriFilePath =
            @"F:\Relationship_Extraction\"
            + "enwikisource-20120808-pages-articles.xml";

        const int MAX_LENGTH = 500;     //sentence的最长值
        
        FileStream fsr, fsw;
        StreamReader sr;
        StreamWriter sw;

        int sent_cnt = 0, text_cnt = 0;       //句子数

        #region Events
        void OnProcessEvent(int va, int vb, int perc)
        {
            if (ProcessEvent != null)
                ProcessEvent(va, vb, perc);
        }

        void OnStateEvent(string info, bool isOver)
        {
            if (StateEvent != null)
                StateEvent(info, isOver);
        }
        #endregion

        #region Generate Sent
        /// <summary>
        /// 中括号替换
        /// </summary>
        /// <param name="text">替换前text</param>
        /// <returns>替换后text</returns>
        string ReplaceBracket(string text)
        {
            string head, mid, tail;
            while (true)
            {
                int lbidx = text.IndexOf("[[");
                int rbidx = text.IndexOf("]]");
                if (lbidx == -1 || rbidx == -1 || rbidx < lbidx)
                    break;
                head = text.Substring(0, lbidx);
                tail = text.Substring(rbidx + 2);
                mid = text.Substring(lbidx + 2, rbidx - lbidx - 2);
                int sptidx = mid.LastIndexOf(@"|");
                if (sptidx != -1)
                    mid = mid.Substring(sptidx + 1);
                text = head + mid + tail;
            }
            return text;
        }

        /// <summary>
        /// 替换转义符
        /// </summary>
        /// <param name="text"></param>
        string ReplaceText(string text)
        {
            text = text.Replace("&lt", "<");
            text = text.Replace("&gt", ">");
            text = text.Replace("&quot", "\"");
            text = text.Replace("&amp", "&");
            return text;
        }

        /// <summary>
        /// 抽取句子核心
        /// </summary>
        /// <param name="text">文字段</param>
        void Ext_Sent_Kernel(string text)
        {
            char[] msk = { '{', '}', ':', ';', 
                             '=', '|', '!', '?', '.' };
            string[] spt = text.Split(msk);
            for (int i = 0; i < spt.Length; i++)
            {
                string s = spt[i];
                s = s.Trim();
                if (s.Length == 0) 
                    continue;
                string[] tmp = s.Split(' ');
                if (tmp.Length < MAX_LENGTH)    //LIMIT LENGTH
                {
                    sw.WriteLine(s.ToLower());
                    sent_cnt++;
                }
                if (sent_cnt % 10000 == 0)
                    OnProcessEvent(sent_cnt, text_cnt, 0);
            }
            //sw.WriteLine(text);
        }

        /// <summary>
        /// 从一段text中抽取句子
        /// </summary>
        /// <param name="text"></param>
        void ExtractSentence(string text)
        {
            if (text.IndexOf("div id") != -1) return;
            if (text.IndexOf("class=") != -1) return;
            if (text.IndexOf("listen|") != -1) return;
            text = ReplaceText(text);
            text = ReplaceBracket(text);
            Ext_Sent_Kernel(text);
        }

        /// <summary>
        /// 获取一个text part
        /// </summary>
        void ExtractTextPart()
        {
            text_cnt = 0;
            string str, txt, st, ed;
            bool fs, fe, flag = false;
            while (!sr.EndOfStream)
            {
                #region Get String
                str = sr.ReadLine();
                str = str.Trim();
                if (str.Length >= 7)
                {
                    st = str.Substring(0, 6);
                    ed = str.Substring(str.Length - 7);
                    fs = (st == @"<text " ? true : false);
                    fe = (ed == @"</text>" ? true : false);
                }
                else
                    fs = fe = false;
                #endregion
                if (fs && fe)
                {
                    #region Line Text
                    flag = true;
                    text_cnt++;
                    int idx = str.IndexOf('>');
                    txt = str.Substring(idx + 1, str.Length - idx - 7);
                    ExtractSentence(txt);
                    flag = false;
                    #endregion
                }
                else if (fs && !fe)
                {
                    #region Text Start
                    flag = true;
                    text_cnt++;
                    int idx = str.IndexOf('>');
                    txt = str.Substring(idx + 1);
                    ExtractSentence(txt);
                    #endregion
                }
                else if (!fs && fe)
                {
                    #region Text End
                    txt = str.Substring(0, str.Length - 7);
                    ExtractSentence(txt); 
                    flag = false;
                    #endregion
                }
                else if (flag == true)
                {
                    #region Text Middle
                    ExtractSentence(str);
                    #endregion
                }
            }
        }

        /// <summary>
        /// 外部调用函数，开始获取句子的一整套工作
        /// </summary>
        public void GenerateSent()
        {
            sent_cnt = 0;
            fsr = new FileStream(oriFilePath, FileMode.Open);
            sr = new StreamReader(fsr, Encoding.UTF8);
            fsw = new FileStream(MainFrame.SentPath, FileMode.Create);
            sw = new StreamWriter(fsw, Encoding.UTF8);
            ExtractTextPart();
            sr.Close(); sw.Close();
            fsr.Close(); fsw.Close();
            OnStateEvent("Extracting sentences complete, with "
                + sent_cnt.ToString() + " sentences.", true);
        }

        #endregion

        public void GenerateSentDs()        //$: punctions  #: illegal words
        {
            int i, l, flag, wds;
            sent_cnt = 0;
            int[] msk = new int[256];
            for (i = 0; i < 9; i++) msk[i + '0'] = 1;
            for (i = 0; i < 26; i++) msk[i + 'a'] = 1;
            msk['-'] = msk['\''] = 1;       //these chars can be seen in a word
            msk[' '] = 2;
            //string YahooSentPath = @"F:\Relationship_Extraction\Data\YahooSent.txt";
            //string YahooSentDsPath = @"F:\Relationship_Extraction\Data\YahooSent_ds.txt";
            fsr = new FileStream(MainFrame.SentPath, FileMode.Open);
            fsw = new FileStream(MainFrame.SentDsPath, FileMode.Create);
            //fsr = new FileStream(YahooSentPath, FileMode.Open);
            //fsw = new FileStream(YahooSentDsPath, FileMode.Create);
            sr = new StreamReader(fsr, Encoding.UTF8);
            sw = new StreamWriter(fsw, Encoding.UTF8);
            while (!sr.EndOfStream)
            {   
                string rd = sr.ReadLine();
                StringBuilder usebd = new StringBuilder();
                StringBuilder wrbd = new StringBuilder();
                
                for (i = 0, l = rd.Length; i < l; i++)  //Punctions are replaced to $
                    if (rd[i] < 256 && msk[rd[i]] == 0)
                        usebd.Append(" $ ");
                    else
                        usebd.Append(rd[i]);
                
                string[] spt = usebd.ToString().Split(' ');
                for (i = wds = 0, l = spt.Length; i < l; i++)
                {
                    string str = spt[i];
                    if (str == "") continue;
                    if (wds++ > 0) wrbd.Append(' ');    //found first word
                    flag = 1;
                    foreach (char c in str)
                        if (c > 256)
                        {
                            flag = 0;
                            break;
                        }
                    if (flag == 1)
                        wrbd.Append(str);
                    else
                        wrbd.Append('#');
                }
                sw.WriteLine(wrbd.ToString());
                if (++sent_cnt % 100000 == 0)
                    OnProcessEvent(sent_cnt, sent_cnt,
                        (int)(100.0 * sent_cnt / MainFrame.TotalSent));
            }
            sr.Close(); sw.Close();
            fsr.Close(); fsw.Close();
            OnStateEvent("Extracting sentences complete, with "
                + sent_cnt.ToString() + " sentences.", true);
        }

        public void GenerateCorpus()
        { 
            int cnt = 0;
            FileStream fsr, fsw;
            fsr = new FileStream(MainFrame.SentPath, FileMode.Open);
            StreamReader sr = new StreamReader(fsr, Encoding.UTF8);
            fsw = new FileStream(MainFrame.CorpusPath, FileMode.Create);
            StreamWriter sw = new StreamWriter(fsw);
            while (!sr.EndOfStream)
            { 
                cnt++;
                string str = sr.ReadLine();
                sw.WriteLine(str + "\t" + cnt.ToString());
                if (cnt % 100000 == 0)
                    OnProcessEvent(MainFrame.TotalSent, cnt, 
                        (int)(100.0 * cnt / MainFrame.TotalSent));
            }
            OnStateEvent("Corpus complete.", true);
        }

        /// <summary>
        /// 文件流关闭
        /// </summary>
        public void CloseFile(int whichWork)
        {
            if (sr != null)
            {
                sr.Close();
                fsr.Close();
            }
            if (sw != null)
            {
                sw.Close();
                fsw.Close();
            }
            if (whichWork == 0)         //Gen Sent
                OnStateEvent("Extracting sentence stopped, with "
                    + sent_cnt.ToString() + " sentences.", true);
            else if (whichWork == 1)    //Gen Sent_ds
                OnStateEvent("Generating Sent_ds stopped.", true);
        }
    }
}
